﻿using MyCrawler.Entity;
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Net;
using System.Text.RegularExpressions;
using System.Web.Script.Serialization;
using Spider.Framework.UtilityCommon.Extensions;

namespace MyCrawler.Utilyties
{
    public class UtilityCtr
    {
        public static int GetTimeLimited()
        {
            return 30; // chạy lần đầu khởi tạo 30 ngày
        }

        /// <summary>
        /// lấy nội dung 1 trang web bình thường, lấy với WebClient.DownloadData
        /// </summary>
        /// <param name="strUrl"></param>
        /// <returns></returns>
        public string GetWebContentTmp(string strUrl)
        {
            string s = "";
            try
            {
                WebClient wClient = new WebClient();
                wClient.Encoding = Encoding.UTF8;
                wClient.Headers.Add("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)");
                
                string strContent = Encoding.UTF8.GetString(wClient.DownloadData(strUrl));

                s = strContent.Replace("\r", "").Replace("\n", "").Replace("\t", "").Replace("\"", "'"); // \r\n\t và \

                string pattMultiWhitepace = @"[ ]{2,}"; // xóa đi chỗ có nhiều khoảng trắng
                s = Regex.Replace(s, pattMultiWhitepace, " "); // xóa nhiều khoảng trắng

            }
            catch (Exception ex) { }
            return s;
        }

        /// <summary>
        /// Dowload html với hàm DownloadString
        /// </summary>
        /// <param name="strUrl"></param>
        /// <returns></returns>
        public static string GetHtmlByDownloadString(string strUrl)
        {
            string s = "";
            try
            {
                using (WebClient wClient = new WebClient())
                {
                    wClient.Encoding = Encoding.UTF8;
                    string strContent = wClient.DownloadString(strUrl);

                    s = strContent.Replace("\r", "").Replace("\n", "").Replace("\t", "").Replace("\"", "'"); // \r\n\t và \

                    string pattMultiWhitepace = @"[ ]{2,}"; // xóa đi chỗ có nhiều khoảng trắng
                    s = Regex.Replace(s, pattMultiWhitepace, " "); // xóa nhiều khoảng trắng
                }

            }
            catch { }
            return s;
        }

        /// <summary>
        /// Lấy nội dung của trang web sử dụng dạng StreamReader - nên dùng cho các trang có https
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public string GetContentWithStreamTmp(string url)
        {
            string s = "";

            try
            {
                using (WebClient client = new WebClient())
                {
                    client.Headers.Add("Accept: text/html, application/xhtml+xml, */*");
                    client.Headers.Add("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.2.12) Gecko/20101026 Firefox/3.6.12");
                    client.Headers.Add("Accept-Language", "en-gb,en;q=0.5");
                    client.Headers.Add("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7");
                    client.Encoding = System.Text.Encoding.UTF8;

                    Stream data = client.OpenRead(url);

                    using (StreamReader reader = new StreamReader(data))
                    {
                        s = reader.ReadToEnd();
                        data.Close();
                        reader.Close();

                        s = s.Replace("\r", "").Replace("\n", "").Replace("\t", "").Replace("\"", "'"); // \r\n\t và \

                        string pattMultiWhitepace = @"[ ]{2,}"; // xóa đi chỗ có nhiều khoảng trắng
                        s = Regex.Replace(s, pattMultiWhitepace, " "); // xóa nhiều khoảng trắng
                    }
                }
            }
            catch (Exception ex) {}
            return s;
        }

        public static string GetHtmls(string url)
        {
            string s = "";

            try
            {
                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
                HttpWebResponse response = (HttpWebResponse)request.GetResponse();
                if (response.StatusCode == HttpStatusCode.OK)
                {
                    Stream receiveStream = response.GetResponseStream();
                    StreamReader readStream = null;
                    if (response.CharacterSet == null)
                    {
                        readStream = new StreamReader(receiveStream);
                    }
                    else
                    {
                        readStream = new StreamReader(receiveStream, Encoding.GetEncoding(response.CharacterSet));
                    }

                    s = readStream.ReadToEnd();

                    response.Close();
                    readStream.Close();
                }
            }
            catch { }

            return s.Trim();
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="remoteUri">ex: http://www.contoso.com/library/homepage/images/</param>
        /// <param name="fileName">ex: ms-banner.gif</param>
        public static void DowloadFile(string link, string path)
        {
            HttpWebRequest webRequest;
            HttpWebResponse webResponse;
            Stream strResponse;
            Stream strLocal;

            using (WebClient wcDownload = new WebClient())
            {
                try
                {
                    // Create a request to the file we are downloading
                    webRequest = (HttpWebRequest)WebRequest.Create(link);

                    // Set default authentication for retrieving the file
                    webRequest.Credentials = CredentialCache.DefaultCredentials;
                    // Retrieve the response from the server
                    webResponse = (HttpWebResponse)webRequest.GetResponse();
                    // Ask the server for the file size and store it
                    Int64 fileSize = webResponse.ContentLength;

                    // Open the URL for download 
                    strResponse = wcDownload.OpenRead(link);

                    // Create a new file stream where we will be saving the data (local drive)
                    strLocal = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.None);

                    // It will store the current number of bytes we retrieved from the server
                    int bytesSize = 0;

                    // A buffer for storing and writing the data retrieved from the server
                    byte[] downBuffer = new byte[2048];

                    // Loop through the buffer until the buffer is empty
                    while ((bytesSize = strResponse.Read(downBuffer, 0, downBuffer.Length)) > 0)
                    {
                        // Write the data from the buffer to the local hard drive
                        strLocal.Write(downBuffer, 0, bytesSize);
                    }

                    webResponse.Close();
                }
                catch
                { }
            }
        }

        /// <summary>
        /// Ghi trạng thái
        /// </summary>
        /// <param name="source"></param>
        /// <param name="blockid"></param>
        /// <param name="message"></param>
        public static void TraceLog_Status(string source, string blockid, string message)
        {
            try
            {
                string pathFolder = @"C:\ToanCrawlerLogs\" + source;
                if (!Directory.Exists(pathFolder))
                {
                    Directory.CreateDirectory(pathFolder);
                }

                #region Kiểm tra và tạo thư mục theo source

                string pattFull = @"C:\ToanCrawlerLogs\" + source + @"\Status";

                if (!Directory.Exists(pattFull))
                {
                    Directory.CreateDirectory(pattFull);
                }
                #endregion

                #region làm việc với file
                // tạo file log nếu chưa tồn tại, file có dạng: nguon_link_trangthaicoloikhong.txt
                string sFile = blockid + ".txt";

                string pathFile = pattFull + "/" + sFile;

                if (!File.Exists(pathFile))
                {
                    File.Create(pathFile).Dispose();
                }
                #endregion

                #region Thực hiện ghi file

                FileStream fsObj = new FileStream(pathFile, FileMode.Append);
                StreamWriter swObj = new StreamWriter(fsObj, Encoding.UTF8);

                // Ghi từng dòng
                swObj.WriteLine(message);

                swObj.Flush();

                swObj.Close();
                fsObj.Close();

                #endregion
            }
            catch (Exception ex) { }
        }

        /// <summary>
        /// Thiết lập thời gian chạy cho 1 blocj
        /// </summary>
        /// <param name="source"></param>
        /// <param name="timeStart"></param>
        /// <param name="totalTopic"></param>
        /// <param name="lastCrawl">thời gian chạy lần trước</param>
        public static void TraceSettingTime(string source, string timeStart, int totalTopic, string lastCrawl)
        {
            try
            {
                string pathFolder = @"C:\ToanCrawlerLogs\" + source + @"\Settings";
                if (!Directory.Exists(pathFolder))
                {
                    Directory.CreateDirectory(pathFolder);
                }

                #region làm việc với file
                string pathFile = pathFolder + "/" + "settingdatestart.txt";

                if (!File.Exists(pathFile))
                {
                    File.Create(pathFile).Dispose();
                }
                #endregion

                #region Thực hiện ghi file

                FileStream fsObj = new FileStream(pathFile, FileMode.Create);
                StreamWriter swObj = new StreamWriter(fsObj, Encoding.UTF8);

                // Ghi từng dòng, với 2 giá trị "thời gian khởi chạy;số topic"
                swObj.WriteLine(timeStart + ";" + totalTopic.ToString() + ";" + lastCrawl);

                swObj.Flush();

                swObj.Close();
                fsObj.Close();

                #endregion
            }
            catch (Exception ex) { }
        }

        public static string ReadSettingTime(string source)
        {
            try
            {
                string pathFile = @"C:\ToanCrawlerLogs\" + source + @"\Settings\settingdatestart.txt";
                if (!File.Exists(pathFile))
                {
                    return "";
                }

                StringBuilder sb = new StringBuilder();
                using (StreamReader sr = new StreamReader(pathFile))
                {
                    String line;
                    while ((line = sr.ReadLine()) != null)
                    {
                        string[] arrValue = line.Split(';');
                        sb.AppendLine(arrValue[0]);
                    }
                }
                return sb.ToString().Trim();
            }
            catch { }
            return "";
        }

        /// <summary>
        /// lấy số topic
        /// </summary>
        /// <param name="source"></param>
        /// <returns></returns>
        public static string ReadTotalTopic(string source)
        {
            try
            {
                string pathFile = @"C:\ToanCrawlerLogs\" + source + @"\Settings\settingdatestart.txt";
                if (!File.Exists(pathFile))
                {
                    return "";
                }

                StringBuilder sb = new StringBuilder();
                using (StreamReader sr = new StreamReader(pathFile))
                {
                    String line;
                    while ((line = sr.ReadLine()) != null)
                    {
                        string[] arrValue = line.Split(';');
                        sb.AppendLine(arrValue[1]);
                    }
                }
                return sb.ToString().Trim();
            }
            catch { }
            return "";
        }

        public static string ReadLastCrawl(string source)
        {
            try
            {
                string pathFile = @"C:\ToanCrawlerLogs\" + source + @"\Settings\settingdatestart.txt";
                if (!File.Exists(pathFile))
                {
                    return "";
                }

                StringBuilder sb = new StringBuilder();
                using (StreamReader sr = new StreamReader(pathFile))
                {
                    String line;
                    while ((line = sr.ReadLine()) != null)
                    {
                        string[] arrValue = line.Split(';');
                        sb.AppendLine(arrValue[2]);
                    }
                }
                return sb.ToString().Trim();
            }
            catch { }
            return "";
        }

        /// <summary>
        /// Đọc các link đã làm việc lưu trong file txt
        /// </summary>
        /// <param name="source"></param>
        /// <returns></returns>
        public static Dictionary<string, long> GetListLinkExisted(string source)
        {
            Dictionary<string, long> dic = new Dictionary<string, long>();
            try
            {
                // mai làm ở đây
                string pathFile = @"C:\ToanCrawlerLogs\" + source + @"\Settings\settinglinks.txt";

                if (File.Exists(pathFile))
                {
                    string txt = File.ReadAllText(pathFile).Trim();

                    if (txt != "")
                    {
                        string[] arrItem = txt.Split(';');

                        if (arrItem.Length > 0)
                        {
                            for (int i = 0; i < arrItem.Length; i++)
                            {
                                string[] arrLink = arrItem[i].Split('|');

                                if (arrLink.Length > 1)
                                {
                                    string link = arrLink[1].Replace("\r", "").Replace("\n", "").Replace("\t", "").Trim();

                                    if (!dic.ContainsKey(link))
                                    {
                                        dic.Add(link, Convert.ToInt64(arrLink[0]));
                                    }
                                }
                            }
                        }
                    }
                }
            }
            catch (Exception ex) { string s = ex.Message; }
            return dic;
        }

        /// <summary>
        /// Ghi link xuống log
        /// </summary>
        /// <param name="source"></param>
        /// <param name="id"></param>
        /// <param name="link"></param>
        public static void WriteLinks(string source, long id, string link)
        {
            try
            {
                string pathFolder = @"C:\ToanCrawlerLogs\" + source + @"\Settings";

                if (!Directory.Exists(pathFolder))
                {
                    Directory.CreateDirectory(pathFolder);
                }

                string pathFile = @"C:\ToanCrawlerLogs\" + source + @"\Settings\settinglinks.txt";
                if (!File.Exists(pathFile))
                {
                    File.Create(pathFile).Dispose();
                }

                #region Thực hiện ghi file
                DateTime dtObj = DateTime.Now;

                //string messageFull = id.ToString() + "|" + link;
                string messageFull = id.ToString() + "|" + link + ";";

                using (FileStream fsObj = new FileStream(pathFile, FileMode.Append))
                {
                    using (StreamWriter swObj = new StreamWriter(fsObj, Encoding.UTF8))
                    {
                        // Ghi từng dòng
                        swObj.Write(messageFull);

                        swObj.Flush();
                    }
                }
                #endregion
            }
            catch (Exception ex) { string s = ex.Message; }
        }

        /// <summary>
        /// Lấy chuỗi kết nối tới database
        /// </summary>
        /// <returns></returns>
        public static string GetConnectString()
        {
            return "Data Source=.;Initial Catalog=CrawlerDb;User ID=coder;Password=coder";
        }

        /// <summary>
        /// Ghi file json
        /// </summary>
        /// <param name="entity"></param>
        /// <returns></returns>
        public static long WriteFileJson(ContentEntity entity)
        {
            try
            {
                string path = @"C:\ToanCrawler\";

                var folder = path + "Data" + "/" + entity.Source + "/" + entity.LastModify.ToString("ddMMyyyy");
                if (!Directory.Exists(folder)) Directory.CreateDirectory(folder);
                entity.Id = entity.LastModify.ToUnixTimestamp();//.ToUnixTimestamp();
                var file = folder + "/" + entity.Id + ".txt";

                if (File.Exists(file)) return 0;
                var sw = new StreamWriter(file);
                sw.Write(entity.ToJson());
                sw.Close();
                return entity.Id;
            }
            catch
            {
                return -1;
            }
        }

        public static long WriteFileJsonInCataLog(string urlCatagories, ContentEntity entity)
        {
            try
            {
                string path = @"C:\ToanCrawler\";

                var folder = path + "Data" + "/" + entity.Source + "/" + urlCatagories.Replace("http://", "") + "/" + entity.LastModify.ToString("ddMMyyyy");
                if (!Directory.Exists(folder)) Directory.CreateDirectory(folder);
                entity.Id = entity.LastModify.ToUnixTimestamp();
                var file = folder + "/" + entity.Id + ".txt";

                if (File.Exists(file)) return 0;
                var sw = new StreamWriter(file);
                sw.Write(entity.ToJson());
                sw.Close();
                return entity.Id;
            }
            catch
            {
                return -1;
            }
        }

        /// <summary>
        /// Lấy danh sách các item theo 1 patten của regular
        /// </summary>
        /// <param name="html"></param>
        /// <param name="pattern"></param>
        /// <returns></returns>
        public static List<string> GetListItem(string html, string pattern)
        {
            List<string> ls = new List<string>();

            foreach (Match m in Regex.Matches(html, pattern))
            {
                ls.Add(m.Value);
            }
            return ls;
        }

        /// <summary>
        /// Lấy item đầu tiên của trong danh sách thỏa mãn 1 pattern
        /// </summary>
        /// <param name="html"></param>
        /// <param name="pattern"></param>
        /// <returns></returns>
        public static string GetItemFirst(string html, string pattern)
        {
            foreach (Match m in Regex.Matches(html, pattern))
            {
                return m.Value;
            }
            return "";
        }

        /// <summary>
        /// Ghi danh sách các link của danh mục ra file json
        /// </summary>
        /// <param name="source"></param>
        /// <param name="entity"></param>
        public static void WriteLinkCatagoriesToJson(string source, CatagoriesToRunEntity entity)
        {
            try
            {
                #region ghi file json

                string pathFile = @"C:\ToanCrawlerLogs\" + source + @"\Settings\settingcatagories.txt";

                if (!Directory.Exists(@"C:\ToanCrawlerLogs\" + source + @"\Settings"))
                {
                    Directory.CreateDirectory(@"C:\ToanCrawlerLogs\" + source + @"\Settings");
                }

                if (!File.Exists(pathFile))
                {
                    File.Create(pathFile).Dispose();
                }

                JavaScriptSerializer jv = new JavaScriptSerializer();
                var json = jv.Serialize(entity);

                using (FileStream fsObj = new FileStream(pathFile, FileMode.Create))
                {
                    using (StreamWriter swObj = new StreamWriter(fsObj, Encoding.UTF8))
                    {
                        // Ghi từng dòng
                        swObj.Write(json);

                        swObj.Flush();

                        swObj.Close();
                        fsObj.Close();
                    }
                }

                #endregion
            }
            catch (Exception ex) { string mes = ex.Message; }
        }

        /// <summary>
        /// Lấy danh sách các link của catagories đã lưu trong file Json
        /// </summary>
        /// <param name="source"></param>
        /// <returns></returns>
        public static CatagoriesToRunEntity ReadLinkCatagoriesFromJson(string source)
        {
            try
            {
                #region đọc file Json
                string pathFile = @"C:\ToanCrawlerLogs\" + source + @"\Settings\settingcatagories.txt";

                string json = File.ReadAllText(pathFile);

                CatagoriesToRunEntity cat = new CatagoriesToRunEntity();
                cat = new System.Web.Script.Serialization.JavaScriptSerializer().Deserialize<CatagoriesToRunEntity>(json);

                return cat;

                #endregion
            }
            catch { }
            return null;
        }

        /// <summary>
        /// chuyển đổi mã Ansi tới unicode
        /// </summary>
        /// <param name="content"></param>
        /// <returns></returns>
        public static string ConvertANSIToUnicode(string content)
        {
            bool hasAnsi = Regex.IsMatch(content, @"\\u[\d]+");

            if (hasAnsi == false)
            {
                return content;
            }

            string s = content.Replace("\\u00e0", "à");
            s = s.Replace("\\u00e1", "á");
            s = s.Replace("\\u1ea3", "ả");
            s = s.Replace("\\u00e3", "ã");
            s = s.Replace("\\u1ea1", "ạ");
            s = s.Replace("\\u0103", "ă");
            s = s.Replace("\\u1eb1", "ằ");
            s = s.Replace("\\u1eaf", "ắ");
            s = s.Replace("\\u1eb3", "ẳ");
            s = s.Replace("\\u1eb5", "ẵ");
            s = s.Replace("\\u1eb7", "ặ");
            s = s.Replace("\\u00e2", "â");
            s = s.Replace("\\u1ea7", "ầ");
            s = s.Replace("\\u1ea5", "ấ");
            s = s.Replace("\\u1ea9", "ẩ");
            s = s.Replace("\\u1eab", "ẫ");
            s = s.Replace("\\u1ead", "ậ");

            s = s.Replace("\\u0111", "đ");
            s = s.Replace("\\u1ead", "ậ");
            s = s.Replace("\\u1ead", "ậ");
            s = s.Replace("\\u1ead", "ậ");
            s = s.Replace("\\u1ead", "ậ");


            s = s.Replace("\\u00e8", "è");
            s = s.Replace("\\u00e9", "é");
            s = s.Replace("\\u1ebb", "ẻ");
            s = s.Replace("\\u1ebd", "ẽ");
            s = s.Replace("\\u1eb9", "ẹ");
            s = s.Replace("\\u00ea", "ê");
            s = s.Replace("\\u1ec1", "ề");
            s = s.Replace("\\u1ebf", "ế");
            s = s.Replace("\\u1ec3", "ể");
            s = s.Replace("\\u1ec5", "ễ");
            s = s.Replace("\\u1ec7", "ệ");

            s = s.Replace("\\u00f2", "ò");
            s = s.Replace("\\u00f3", "ó");
            s = s.Replace("\\u1ecf", "ỏ");
            s = s.Replace("\\u00f5", "õ");
            s = s.Replace("\\u1ecd", "ọ");
            s = s.Replace("\\u00f4", "ô");
            s = s.Replace("\\u1ed3", "ồ");
            s = s.Replace("\\u1ed1", "ố");
            s = s.Replace("\\u1ed5", "ổ");
            s = s.Replace("\\u1ed7", "ỗ");
            s = s.Replace("\\u1ed9", "ộ");
            s = s.Replace("\\u01a1", "ơ");
            s = s.Replace("\\u1edd", "ờ");
            s = s.Replace("\\u1edb", "ớ");
            s = s.Replace("\\u1edf", "ở");
            s = s.Replace("\\u1ee1", "ỡ");
            s = s.Replace("\\u1ee3", "ợ");

            s = s.Replace("\\u00f9", "ù");
            s = s.Replace("\\u00fa", "ú");
            s = s.Replace("\\u1ee7", "ủ");
            s = s.Replace("\\u0169", "ũ");
            s = s.Replace("\\u1ee5", "ụ");
            s = s.Replace("\\u01b0", "ư");
            s = s.Replace("\\u1eeb", "ừ");
            s = s.Replace("\\u1ee9", "ứ");
            s = s.Replace("\\u1eed", "ử");
            s = s.Replace("\\u1eef", "ữ");
            s = s.Replace("\\u1ef1", "ự");

            s = s.Replace("\\u00c0", "À");
            s = s.Replace("\\u00c1", "Á");
            s = s.Replace("\\u1ea2", "Ả");
            s = s.Replace("\\u00c3", "Ã");
            s = s.Replace("\\u1ea0", "Ạ");
            s = s.Replace("\\u0102", "Â");
            s = s.Replace("\\u1eb0", "Ầ");
            s = s.Replace("\\u1eae", "Ấ");
            s = s.Replace("\\u1eb2", "Ẩ");
            s = s.Replace("\\u1eb4", "Ẫ");
            s = s.Replace("\\u1eb6", "Ậ");
            s = s.Replace("\\u00c2", "Ă");
            s = s.Replace("\\u1ea6", "Ằ");
            s = s.Replace("\\u1ea4", "Ắ");
            s = s.Replace("\\u1ea8", "Ẳ");
            s = s.Replace("\\u1eaa", "Ẵ");
            s = s.Replace("\\u1eac", "Ặ");

            s = s.Replace("\\u0110", "Đ");

            s = s.Replace("\\u00c8", "È");
            s = s.Replace("\\u00c9", "É");
            s = s.Replace("\\u1eba", "Ẻ");
            s = s.Replace("\\u1ebc", "Ẽ");
            s = s.Replace("\\u1eb8", "Ẹ");
            s = s.Replace("\\u00ca", "Ê");
            s = s.Replace("\\u1ec0", "Ề");
            s = s.Replace("\\u1ebe", "Ế");
            s = s.Replace("\\u1ec2", "Ể");
            s = s.Replace("\\u1ec4", "Ễ");
            s = s.Replace("\\u1ec6", "Ệ");

            s = s.Replace("\\u00d2", "Ò");
            s = s.Replace("\\u00d3", "Ó");
            s = s.Replace("\\u1ece", "Ỏ");
            s = s.Replace("\\u00d5", "Õ");
            s = s.Replace("\\u1ec6", "Ọ");
            s = s.Replace("\\u00d4", "Ô");
            s = s.Replace("\\u1ed2", "Ồ");
            s = s.Replace("\\u1ed0", "Ố");
            s = s.Replace("\\u1ed4", "Ổ");
            s = s.Replace("\\u1ed6", "Ỗ");
            s = s.Replace("\\u1ed8", "Ộ");
            s = s.Replace("\\u01a0", "Ơ");
            s = s.Replace("\\u1edc", "Ờ");
            s = s.Replace("\\u1eda", "Ớ");
            s = s.Replace("\\u1ede", "Ở");
            s = s.Replace("\\u1ee0", "Ỡ");
            s = s.Replace("\\u1ee2", "Ợ");

            s = s.Replace("\\u00d9", "Ù");
            s = s.Replace("\\u00da", "Ú");
            s = s.Replace("\\u1ee6", "Ủ");
            s = s.Replace("\\u0168", "Ũ");
            s = s.Replace("\\u1ee4", "Ụ");
            s = s.Replace("\\u01af", "Ư");
            s = s.Replace("\\u1eea", "Ừ");
            s = s.Replace("\\u1ee8", "Ứ");
            s = s.Replace("\\u1eec", "Ử");
            s = s.Replace("\\u1eee", "Ữ");
            s = s.Replace("\\u1ef0", "Ự");

            s = s.Replace("\\u00ec", "ì");
            s = s.Replace("\\u00ed", "í");
            s = s.Replace("\\u1ec9", "ỉ");
            s = s.Replace("\\u0129", "ĩ");
            s = s.Replace("\\u1ecb", "ị");
            s = s.Replace("\\u00cc", "Ì");
            s = s.Replace("\\u00cd", "Í");
            s = s.Replace("\\u1ec8", "Ỉ");
            s = s.Replace("\\u0128", "Ĩ");
            s = s.Replace("\\u1eca", "Ị");

            s = s.Replace("\\u1ef3", "ỳ");
            s = s.Replace("\\u00fd", "ý");
            s = s.Replace("\\u1ef7", "ỷ");
            s = s.Replace("\\u1ef9", "ỹ");
            s = s.Replace("\\u1ef5", "ỵ");
            s = s.Replace("\\u1ef2", "Ỳ");
            s = s.Replace("\\u00dd", "Ý");
            s = s.Replace("\\u1ef6", "Ỷ");
            s = s.Replace("\\u1ef8", "Ỹ:");
            s = s.Replace("\\u1ef4", "Ỵ");

            #region [dấu sắc]
            s = s.Replace("a\\u0301", "á");
            s = s.Replace("ă\\u0301", "ắ");
            s = s.Replace("â\\u0301", "ấ");
            s = s.Replace("e\\u0301", "é");
            s = s.Replace("ê\\u0301", "ế");
            s = s.Replace("i\\u0301", "í");
            s = s.Replace("y\\u0301", "ý");
            s = s.Replace("o\\u0301", "ó");
            s = s.Replace("ô\\u0301", "ố");
            s = s.Replace("u\\u0301", "ú");
            s = s.Replace("ư\\u0301", "ứ");

            s = s.Replace("A\\u0301", "Á");
            s = s.Replace("Ă\\u0301", "Ắ");
            s = s.Replace("Â\\u0301", "Ấ");
            s = s.Replace("E\\u0301", "É");
            s = s.Replace("Ê\\u0301", "Ế");
            s = s.Replace("I\\u0301", "Í");
            s = s.Replace("Y\\u0301", "Ý");
            s = s.Replace("O\\u0301", "Ó");
            s = s.Replace("Ô\\u0301", "Ố");
            s = s.Replace("U\\u0301", "Ú");
            s = s.Replace("Ư\\u0301", "Ứ");
            #endregion

            #region [dấu huyền]
            s = s.Replace("a\\u0300", "à");
            s = s.Replace("ă\\u0300", "ằ");
            s = s.Replace("â\\u0300", "ầ");
            s = s.Replace("e\\u0300", "è");
            s = s.Replace("ê\\u0300", "ề");
            s = s.Replace("i\\u0300", "ì");
            s = s.Replace("y\\u0300", "ỳ");
            s = s.Replace("o\\u0300", "ò");
            s = s.Replace("ô\\u0300", "ồ");
            s = s.Replace("ơ\\u0300", "ờ");
            s = s.Replace("u\\u0300", "ù");
            s = s.Replace("ư\\u0300", "ừ");

            s = s.Replace("A\\u0300", "À");
            s = s.Replace("Ă\\u0300", "Ằ");
            s = s.Replace("Â\\u0300", "Ầ");
            s = s.Replace("E\\u0300", "È");
            s = s.Replace("Ê\\u0300", "Ề");
            s = s.Replace("I\\u0300", "Ì");
            s = s.Replace("Y\\u0300", "Ỳ");
            s = s.Replace("O\\u0300", "Ò");
            s = s.Replace("Ô\\u0300", "Ồ");
            s = s.Replace("Ơ\\u0300", "Ờ");
            s = s.Replace("U\\u0300", "Ù");
            s = s.Replace("Ư\\u0300", "Ừ");
            #endregion

            #region [dấu hoi]
            s = s.Replace("a\\u0309", "ả");
            s = s.Replace("ă\\u0309", "ẳ");
            s = s.Replace("â\\u0309", "ẩ");
            s = s.Replace("e\\u0309", "ẻ");
            s = s.Replace("ê\\u0309", "ể");
            s = s.Replace("i\\u0309", "ỉ");
            s = s.Replace("y\\u0309", "ỷ");
            s = s.Replace("o\\u0309", "ỏ");
            s = s.Replace("ô\\u0309", "ổ");
            s = s.Replace("u\\u0309", "ủ");
            s = s.Replace("ư\\u0309", "ử");

            s = s.Replace("A\\u0309", "Ả");
            s = s.Replace("Ă\\u0309", "Ẳ");
            s = s.Replace("Â\\u0309", "Ẩ");
            s = s.Replace("E\\u0309", "Ể");
            s = s.Replace("Ê\\u0309", "Ể");
            s = s.Replace("I\\u0309", "Ỉ");
            s = s.Replace("Y\\u0309", "Ỷ");
            s = s.Replace("O\\u0309", "Ỏ");
            s = s.Replace("Ô\\u0309", "Ổ");
            s = s.Replace("U\\u0309", "Ủ");
            s = s.Replace("Ư\\u0309", "Ử");
            #endregion

            #region [dấu ngã]
            s = s.Replace("a\\u0303", "ã");
            s = s.Replace("ă\\u0303", "ẵ");
            s = s.Replace("â\\u0303", "ẫ");
            s = s.Replace("e\\u0303", "ẽ");
            s = s.Replace("ê\\u0303", "ễ");
            s = s.Replace("i\\u0303", "ĩ");
            s = s.Replace("y\\u0303", "ỹ");
            s = s.Replace("o\\u0303", "õ");
            s = s.Replace("ô\\u0303", "ỗ");
            s = s.Replace("u\\u0303", "ũ");
            s = s.Replace("ư\\u0303", "ữ");

            s = s.Replace("A\\u0303", "Ã");
            s = s.Replace("Ă\\u0303", "Ẵ");
            s = s.Replace("Â\\u0303", "Ẫ");
            s = s.Replace("E\\u0303", "Ẽ");
            s = s.Replace("Ê\\u0303", "Ễ");
            s = s.Replace("I\\u0303", "Ĩ");
            s = s.Replace("Y\\u0303", "Ỹ");
            s = s.Replace("O\\u0303", "Õ");
            s = s.Replace("Ô\\u0303", "Ỗ");
            s = s.Replace("U\\u0303", "Ũ");
            s = s.Replace("Ư\\u0303", "Ữ");
            #endregion

            #region [dấu nặng]
            s = s.Replace("a\\u0323", "ạ");
            s = s.Replace("ă\\u0323", "ặ");
            s = s.Replace("â\\u0323", "ậ");
            s = s.Replace("e\\u0323", "ẹ");
            s = s.Replace("ê\\u0323", "ệ");
            s = s.Replace("i\\u0323", "ị");
            s = s.Replace("y\\u0323", "ỵ");
            s = s.Replace("o\\u0323", "ọ");
            s = s.Replace("ô\\u0323", "ộ");
            s = s.Replace("u\\u0323", "ụ");
            s = s.Replace("ư\\u0323", "ự");

            s = s.Replace("A\\u0323", "Ạ");
            s = s.Replace("Ă\\u0323", "Ặ");
            s = s.Replace("Â\\u0323", "Ậ");
            s = s.Replace("E\\u0323", "Ẹ");
            s = s.Replace("Ê\\u0323", "Ệ");
            s = s.Replace("I\\u0323", "Ị");
            s = s.Replace("Y\\u0323", "Ỵ");
            s = s.Replace("O\\u0323", "Ọ");
            s = s.Replace("Ô\\u0323", "Ộ");
            s = s.Replace("U\\u0323", "Ụ");
            s = s.Replace("Ư\\u0323", "Ự");
            #endregion

            #region [Khoảng trắng]
            s = s.Replace("&nbsp;", " ");
            #endregion
            return s;
        }

        public static string ConvertNCRDecimalToUnicode(string content)
        {
            // kiểm tra chuỗi có ký tự NCRDecimal ko?
            bool hasNcr = Regex.IsMatch(content, @"#[\d]+;");

            if (hasNcr == false)
            {
                return content;
            }

            string s = content.Replace("&#97;", "a");
            s = s.Replace("&#225;", "á");
            s = s.Replace("&#224;", "à");
            s = s.Replace("&#7843;", "ả");
            s = s.Replace("&#227;", "ã");
            s = s.Replace("&#7841;", "ạ");

            s = s.Replace("&#259;", "ă");
            s = s.Replace("&#7855;", "ắ");
            s = s.Replace("&#7857;", "ằ");
            s = s.Replace("&#7859;", "ẳ");
            s = s.Replace("&#7861;", "ẵ");
            s = s.Replace("&#7863;", "ặ");

            s = s.Replace("&#226;", "â");
            s = s.Replace("&#7845;", "ấ");
            s = s.Replace("&#7847;", "ầ");
            s = s.Replace("&#7849;", "ẩ");
            s = s.Replace("&#7851;", "ẵ");
            s = s.Replace("&#7853;", "ậ");

            s = s.Replace("&#101;", "e");
            s = s.Replace("&#233;", "é");
            s = s.Replace("&#232;", "è");
            s = s.Replace("&#7867;", "ẻ");
            s = s.Replace("&#7869;", "ẽ");
            s = s.Replace("&#7865;", "ẹ");

            s = s.Replace("&#234;", "ê");
            s = s.Replace("&#7871;", "ế");
            s = s.Replace("&#7873;", "ề");
            s = s.Replace("&#7875;", "ể");
            s = s.Replace("&#7877;", "ễ");
            s = s.Replace("&#7879;", "ệ");

            s = s.Replace("&#105;", "i");
            s = s.Replace("&#237;", "í");
            s = s.Replace("&#236;", "ì");
            s = s.Replace("&#7881;", "ỉ");
            s = s.Replace("&#297;", "ĩ");
            s = s.Replace("&#7883;", "ị");

            s = s.Replace("&#111;", "o");
            s = s.Replace("&#243;", "ó");
            s = s.Replace("&#242;", "ò");
            s = s.Replace("&#7887;", "ỏ");
            s = s.Replace("&#245;", "õ");
            s = s.Replace("&#7885;", "ọ");

            s = s.Replace("&#244;	", "ô");
            s = s.Replace("&#7889;", "ố");
            s = s.Replace("&#7891;", "ồ");
            s = s.Replace("&#7893;", "ổ");
            s = s.Replace("&#7895;", "ỗ");
            s = s.Replace("&#7897;", "ộ");

            s = s.Replace("&#417;", "ơ");
            s = s.Replace("&#7899;", "ớ");
            s = s.Replace("&#7901;", "ờ");
            s = s.Replace("&#7903;", "ở");
            s = s.Replace("&#7905;", "ỡ");
            s = s.Replace("&#7907;", "ợ");

            s = s.Replace("&#117;", "u");
            s = s.Replace("&#250;", "ú");
            s = s.Replace("&#249;", "ù");
            s = s.Replace("&#7911;", "ủ");
            s = s.Replace("&#361;", "ũ");
            s = s.Replace("&#7909;", "ụ");

            s = s.Replace("&#432;", "y");
            s = s.Replace("&#253;", "ý");
            s = s.Replace("&#7923;", "ỳ");
            s = s.Replace("&#7927;", "ỷ");
            s = s.Replace("&#7927;", "ỹ");
            s = s.Replace("&#7925;", "ỵ");

            s = s.Replace("&#273;", "đ");

            s = s.Replace("&#65;", "A");
            s = s.Replace("&#193;", "Á");
            s = s.Replace("&#192;", "À");
            s = s.Replace("&#7842;", "Ả");
            s = s.Replace("&#195;", "Ã");
            s = s.Replace("&#7840;", "Ạ");

            s = s.Replace("&#258;", "Ă");
            s = s.Replace("&#7854;", "Ắ");
            s = s.Replace("&#7856;", "Ằ");
            s = s.Replace("&#7858;", "Ẳ");
            s = s.Replace("&#7860;", "Ẵ");
            s = s.Replace("&#7862;", "Ặ");

            s = s.Replace("&#194;", "Â");
            s = s.Replace("&#7844;", "Ấ");
            s = s.Replace("&#7846;", "Ầ");
            s = s.Replace("&#7848;", "Ẩ");
            s = s.Replace("&#7850;", "Ẫ");
            s = s.Replace("&#7852;", "Ậ");

            s = s.Replace("&#69;", "E");
            s = s.Replace("&#201;", "É");
            s = s.Replace("&#200;", "È");
            s = s.Replace("&#7866;", "Ẻ");
            s = s.Replace("&#7868;", "Ẽ");
            s = s.Replace("&#7864;", "Ẹ");

            s = s.Replace("&#202;", "Ê");
            s = s.Replace("&#7870; ", "Ế");
            s = s.Replace("&#7872;", "Ề");
            s = s.Replace("&#7874;", "Ể");
            s = s.Replace("&#7876;", "Ễ");
            s = s.Replace("&#7878;", "Ệ");

            s = s.Replace("&#73;", "I");
            s = s.Replace("&#205;", "Í");
            s = s.Replace("&#204;", "Ì");
            s = s.Replace("&#7880;", "Ỉ");
            s = s.Replace("&#296;", "Ĩ");
            s = s.Replace("&#7882;", "Ị");

            s = s.Replace("&#79;", "O");
            s = s.Replace("&#211;", "Ó");
            s = s.Replace("&#210;", "Ò");
            s = s.Replace("&#7886;", "Ỏ");
            s = s.Replace("&#213;", "Õ");
            s = s.Replace("&#7884;", "Ọ");

            s = s.Replace("&#212;", "Ô");
            s = s.Replace("&#7888;", "Ố");
            s = s.Replace("&#7890;", "Ồ");
            s = s.Replace("&#7892;", "Ổ");
            s = s.Replace("&#7894;", "Ỗ");
            s = s.Replace("&#7896;", "Ộ");

            s = s.Replace("&#416;", "Ơ");
            s = s.Replace("&#7898;", "Ớ");
            s = s.Replace("&#7900;", "Ờ");
            s = s.Replace("&#7902;", "Ở");
            s = s.Replace("&#7904;", "Ỡ");
            s = s.Replace("&#7906;", "Ợ");

            s = s.Replace("&#85;", "U");
            s = s.Replace("&#218;", "Ú");
            s = s.Replace("&#217;", "Ù");
            s = s.Replace("&#7910;", "Ủ");
            s = s.Replace("&#360;", "Ũ");
            s = s.Replace("&#7908;", "Ụ");

            s = s.Replace("&#431;", "Ư");
            s = s.Replace("&#7912;", "Ứ");
            s = s.Replace("&#7914;", "Ừ");
            s = s.Replace("&#7916;", "Ử");
            s = s.Replace("&#7918;", "Ữ");
            s = s.Replace("&#7920;", "Ự");

            s = s.Replace("&#89;", "Y");
            s = s.Replace("&#221;", "Ý");
            s = s.Replace("&#7922;", "Ỳ");
            s = s.Replace("&#7926;", "Ỷ");
            s = s.Replace("&#7928;", "Ỹ");
            s = s.Replace("&#7924;", "Ỵ");

            s = s.Replace("&#272;", "Đ");

            return s;
        }

        /// <summary>
        /// Replace Unicode
        /// </summary>
        /// <param name="str_input">chuỗi đầu vào</param>
        /// <returns>chuỗi sạch unicode</returns>
        public static string ReplaceUnicode(string str_input)
        {
            try
            {
                Regex regex = new Regex("\\p{IsCombiningDiacriticalMarks}+");
                string temp = str_input.Normalize(NormalizationForm.FormD);
                temp = regex.Replace(temp, String.Empty).Replace('\u0111', 'd').Replace('\u0110', 'D');

                string partSpec = @"[!@#$%^&*()-=_,.\/\\;|:\?\[\]–']";
                str_input = Regex.Replace(temp, partSpec, "").Replace("\"", "");
                str_input = Regex.Replace(str_input, @"[\s]{1,}", "-");
                str_input = Regex.Replace(str_input, @"[-]{1,}", "-");
            }
            catch
            {
                str_input = "";
            }
            return str_input;

        }

        /// <summary>
        /// Ghi log file Json thông qua Ftp
        /// </summary>
        /// <param name="entity"></param>
        /// <returns></returns>
        public string LogFileJson(ContentEntity entity)
        {
            try
            {
                string mes = "";
                var folder = Config.FTP_FOLDER + "/" + entity.Source + "/" + entity.LastModify.ToString("ddMMyyyy");
                if (!Directory.Exists(folder)) Directory.CreateDirectory(folder);
                var file = folder + "/" + entity.LastModify.ToUnixTimestamp() + ".txt";

                if (File.Exists(file))
                {
                    mes = file + ", file existed";
                    return mes;
                }

                var sw = new StreamWriter(file);
                sw.Write(entity.ToJson());
                sw.Close();

                /* -- tạm tắt 
                if (File.Exists(file))
                {
                    FtpExpress ftp = new FtpExpress(Config.FTP_ADDRESS, Config.FTP_USER, Config.FTP_PASSWORD);
                    bool re = ftp.Upload(file, file, false);

                    if (re == true)
                    {
                        mes = file + ", created new & upload success";
                    }
                    else
                    {
                        mes = file + ", created new & upload false";
                    }
                }
                 */
                return mes;
            }
            catch (Exception ex)
            {
                return "error: " + ex.Message;
            }
        }
    }
}
